import json
import random
import time
import bs4
import requests
from trafilatura import extract
from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync

# response = requests.get(
#     "https://www.sciencedirect.com/science/article/pii/S2666449625000611")

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False)
    context = browser.new_context()
    page = context.new_page()
    # ✨ 应用 stealth 技术
    stealth_sync(page)

    page.goto(
        "https://sam.gov/workspace/contract/opp/9a4897fd0111410eaf21460c996cac04/view",
        timeout=200000,
        wait_until="networkidle",
    )
    time.sleep(random.uniform(3, 4))
    page.wait_for_load_state("networkidle")

    response = page.content()
# class="grid-row margin-top-2 ng-star-inserted"
soup = bs4.BeautifulSoup(response, "html.parser")
grid_rows = soup.find("div", class_="grid-row margin-top-2 ng-star-inserted")

from markdownify import markdownify as md
text = md(grid_rows.prettify())

with open("output.md", "w", encoding="utf-8") as f:
    f.write(text)
